pub type TokenID = u32;

pub trait Tokenize {
    fn encode(&self, text: &str) -> Vec<TokenID>;
    fn decode(&self, ids: &[TokenID]) -> String;
}